#!/usr/bin/perl
use Cwd;

$baseurl = getcwd;

if ($ARGV[0] eq 'CREATING') {
    $mode = 'CREATING';
} else {
    $mode = 'UPDATING';
}

if ($ARGV[1] =~ /\//) {
    ($baseurl = $ARGV[1]) =~ s/[\/]*$//;
}

$error_outs = $baseurl . '/tocs_970s_XML_parser_errors.txt';
$print_outs = $baseurl . '/tocs_970s_XML_parser_prints.txt';

binmode(STDOUT, ':utf8');

open (STDERR, ">$error_outs");
open (STDOUT, ">$print_outs");

print "baseurl: " . $baseurl . "\n";

# local system variables
$connect = $baseurl . '/connect.pl';
require $connect;

use IO::File;

#use Data::Dumper;

use URI::Escape;

use LWP::Simple;

use XML::Parser;

use utf8;

use Unicode::Normalize;

use DBI;

$diacritics_hashfile = $baseurl . '/hashfiles/diacritics_hash.pl';

$diacritics_loader = IO::File->new("> $diacritics_hashfile") || die "cannot open $diacritics_hashfile for writing: $!\n";
binmode( $diacritics_loader, ':utf8' );
flock($diacritics_loader, 2);

    print $diacritics_loader "%diacritics_hash = ( \n";
foreach $first (0...9, A...F) {
    foreach $second (0...9, A...F) {
        foreach $third (0...9, A...F) {
            foreach $fourth (0...9, A...F) {
                $character = join "", $first, $second, $third, $fourth;
                    if($character =~ /FE20|FE22/) {
                        print $diacritics_loader "\"$character\" => \" \", \n";
                    } elsif($character =~ /FE21/) {
                        print $diacritics_loader "\"$character\" => \"\\x{0361}\", \n";
                    } elsif($character =~ /FE23/) {
                        print $diacritics_loader "\"$character\" => \"\\x{0360}\", \n";
                    } else {
                        print $diacritics_loader "\"$character\" => \"\\x{" .  $character . "}\", \n";
                    }
            }
        }
    }
}
    print $diacritics_loader "); \n";

        $diacritics_loader->autoflush(1);
        close($diacritics_loader, 2) || "cannot close $diacritics_loader: $!";

&hashes_and_arrays; #SUBROUTINE AT END OF SCRIPT

$filecount = 0;
$previous_time = '';

$record_separator = '\\n';

# connection for the select
$dbh = DBI->connect("DBI:mysql:database=$db_name;host=$db_host", $db_user, $db_pass) or die "couldn't connect to database: " . DBI->errstr;

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$start_time = "$HOUR:$MIN:$SEC";

sub uniqueness {
    my $label = $_[1];
    (my $content = $_[2]) =~ s/\.[ ]*$|\.,[ ]*$|,[ ]*$//g;

    if($label eq '') {
    $term = join '', $label, $content;
    } else {
    $label =~ s/ $//;
    $term = join '\\t', $label, $content;
    }
    
    if($query_insert{$_[0]} !~ /(^|\Q$record_separator\E)\Q$term\E($|\Q$record_separator\E)/) {
        if($query_insert{$_[0]} ne '') {
            $query_insert{$_[0]} .= $record_separator;
        }

        $query_insert{$_[0]} .= $term;
    }
}

#*****************

$folder_name = 'tocs_970s_bibnumbers';

$where = $baseurl . '/tocs_970s_bibnumbers';

while (defined($next_file = <$where/*.out>)) {

print "filename: " . $next_file . "\n";

    open(IN, "< $next_file") || die "cannot open $next_file for reading: $!";

        while(<IN>) {

            if($_ !~ /^RECORD #\(BIBLIO\)/) {

$bib_number = substr $_, 0, 8;
$xml_file = $baseurl . '/tocs_970s_XML_files/' . $bib_number . '.xml';
$url = $xrecord_baseurl . $bib_number;

$cp = get $url;

        $xml_file = $baseurl . '/tocs_970s_XML_files/' . $bib_number . '.xml';

        $batch_loader = IO::File->new("> $xml_file") || die "cannot open $xml_file for writing: $!\n";
        binmode( $batch_loader, ':utf8' );
        flock($batch_loader, 2);

        print $batch_loader $cp . "\n";

        $batch_loader->autoflush(1);
        close($batch_loader, 2) || "cannot close $batch_loader: $!";

            }

        }

unlink($next_file);

}


#*****************

$where = $baseurl . '/tocs_970s_XML_files';

while (defined($next_file = <$where/*.xml>)) {

    $edit1 = "perl -pi -e 's/[\x1b]|[\x1d]|[\x1e]|[\x1f]//g' $next_file";
    system($edit1);

    my $int = $filecount / 1000;

    if ($int =~ m/^\d+$/) {

        if($filecount > 0) {
            $batch_loader->autoflush(1);
            close($batch_loader, 2) || "cannot close $batch_loader: $!";
        }

        $batch_textfiles = $baseurl . '/tocs_970s_textfiles/tocs_970s_' . $filecount . ".txt";

        $batch_loader = IO::File->new("> $batch_textfiles") || die "cannot open $batch_textfiles for writing: $!\n";
        binmode( $batch_loader, ':utf8' );
        flock($batch_loader, 2);
    }

$filecount++;

%query_insert = (
  local_control_number => $institution,
  authors_tocs => '',
  titles_tocs => '',
  tocs => ''
);

my $CHAPTER = '';
my $TITLE = '';
my @AUTHOR = ();
my @AUTHOR_STRING = ();
my $FINAL_AUTHOR_STRING = '';
my $PAGES = '';
my $marctag = '';
my $subfielddata = '';


  my $parser = new XML::Parser ( Handlers => {   # Creates our parser object
                              Start   => \&hdl_start,
                              End     => \&hdl_end,
                              Char    => \&hdl_char,
                              Default => \&hdl_def,
                            });
  $parser->parsefile($next_file, forcearray => 1);

  # The Handlers
  sub hdl_start{
      my ($p, $element, %atts) = @_;
        if($element eq 'RECORDKEY') {
            $tag = 'RECORDKEY';
        } elsif($element eq 'MARCTAG') {
            $tag = 'MARCTAG';
        } elsif($element eq 'SUBFIELDINDICATOR') {
            $tag = 'SUBFIELDINDICATOR';
        } elsif($element eq 'SUBFIELDDATA') {
            $tag = 'SUBFIELDDATA';
        }
  }
   
  sub hdl_end{
      my ($p, $element) = @_;
        $tag = '';
        if($element eq 'RECORDKEY') {
            $query_insert{'local_control_number'} .= "$recordkey";
        } 
        
        if($element eq 'VARFLD') {
            if($marctag eq '970') {
                if($TITLE ne '') {
                    uniqueness('titles_tocs', '', $TITLE);
                }

                for ($f = 0; $f <= $#AUTHOR; $f++) {

                    if($f == 0) {
                        $FINAL_AUTHOR_STRING = " / ";
                    }

                    uniqueness('authors_tocs', '', $AUTHOR[$f]);
#                    $AUTHOR_url = uri_escape($AUTHOR[$f]);

                    $FINAL_AUTHOR_STRING .= "<a href=\"" . $AUTHOR[$f] . "\">" . $AUTHOR_STRING[$f] . "</a>";

                    if($f >= 0 && $f < $#AUTHOR) {
                        $FINAL_AUTHOR_STRING .= ", ";
                    }

                }


                if($CHAPTER ne '') {
                    $CHAPTER .= ". ";
                }

                if($PAGES ne '') {
                    $PAGES = " " . $PAGES;
                }
                
                $TOCS = $CHAPTER . '\t' . $TITLE . '\t' . $FINAL_AUTHOR_STRING . '\t' . $PAGES;

                uniqueness('tocs', '', $TOCS);
            }

            $CHAPTER = '';
            $TITLE = '';
            @AUTHOR = ();
            @AUTHOR_STRING = ();
            $FINAL_AUTHOR_STRING = '';
            $PAGES = '';

            $marctag = '';
            $subfielddata = '';
        }
  }
  
  sub hdl_char {
      my ($p, $str) = @_;
        if($tag eq 'RECORDKEY') {
            $recordkey = $str;
        } elsif($tag eq 'MARCTAG') {
            $marctag = $str;
        } elsif($tag eq 'SUBFIELDINDICATOR') {
            $subfieldindicator = $str;
        } elsif($tag eq 'SUBFIELDDATA') {
            if($marctag eq '970') {
                if($subfieldindicator eq 'l') {
                    $CHAPTER .= $str;
                } elsif($subfieldindicator eq 't') {
                    $TITLE .= $str;
                } elsif($subfieldindicator eq 'c') { 
                    push(@AUTHOR_STRING, $str);
                } elsif($subfieldindicator eq 'f') {
                    push(@AUTHOR, $str);
                } elsif($subfieldindicator eq 'p') {
                    $PAGES .= $str;
                }
            }
        }
  }
  
  sub hdl_def { }  # We just throw everything else away
  
#placement for the insert into bibs_record_table

@keys = keys %query_insert;
@values = values %query_insert;

$mysqlkeys = join(", ", @keys);
my $mysqlvalues = join("\t", @values);
$mysqlvalues =~ s/\\\t/\\ \t/g;

my %utfs = '';
while ($mysqlvalues =~ /\{u([0-9A-Z]{4})\}/g) {
    $utfs{$1}++;
}
        foreach my $power (keys %utfs) {
            $mysqlvalues =~ s/{u$power}/$diacritics_hash{$power}/g;
        }

print $batch_loader $mysqlvalues . "\n";

#undef %query_insert;
#undef $mysqlvalues;

$this_time = time;
$time_diff = $this_time - $previous_time;
$total_time += $time_diff;
$number_of_records++;
$total_number_of_records++;
$total_number_of_files++;
if($time_diff > 0) {
$time_diff = $time_diff / 60;
$time_diff =~ s/\./:/;
print "total_time: $total_time -- $time_diff ($number_of_records) :  $next_file \n";
$number_of_records = 0;
}
$previous_time = "$this_time";

unlink $next_file;

}
        $batch_loader->autoflush(1);
        close($batch_loader, 2) || "cannot close $batch_loader: $!";


($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$end_of_file_creation_time = "$HOUR:$MIN:$SEC";

$sth2 = $dbh->do("TRUNCATE TABLE $db_local_tables_hash{'tocs_970s_table'}");

$this_table = $db_local_tables_hash{'tocs_970s_table'};
$that_table = $db_local_tables_hash{'bibs_record_table'};

$where = $baseurl . '/tocs_970s_textfiles';

while (defined($next_file = <$where/*.txt>)) {

    $sth3 = $dbh->do("LOAD DATA INFILE '" . $next_file . "' REPLACE INTO TABLE $this_table FIELDS TERMINATED BY '\t' (" . $mysqlkeys . ")") || print "died again";

unlink($next_file);

}

#($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
#$that_table_new_update_time = "$HOUR:$MIN:$SEC";
#
#        $querystring = "UPDATE $that_table, $this_table
#            SET $that_table.authors_tocs = $this_table.authors_tocs, 
#                $that_table.titles_tocs = $this_table.titles_tocs,
#                $that_table.tocs = $this_table.tocs
#            WHERE $that_table.local_control_number = $this_table.local_control_number";
#
#        $query = $dbh->prepare($querystring)
#            or die("couldn't prepare statement: ". $dbh->errstr);
#
#        $query->execute()
#            or die("couldn't execute statement: ". $dbh->errstr);


$dbh->disconnect;

($HOUR, $MIN, $SEC) = (localtime)[2, 1, 0];
$end_time = "$HOUR:$MIN:$SEC";

if($total_time > 0) {
$total_time = $total_time / 60;
$total_time =~ s/\./:/;
}

print "Start time: " . $start_time . "\nEnd of file creation time: " . $end_of_file_creation_time . "\n $that_table update time" . $that_table_update_time . "\nEnd time: " . $end_time . "\n Number of files: " . $total_number_of_files . "\n Total time in minutes: " . $total_time . "\n";

close (STDERR) || "cannot close errors.txt: $!";
close (STDOUT) || "cannot close prints.txt: $!";

#SUBROUTINES

sub hashes_and_arrays {

$hash_files = $baseurl . '/hashfiles';

while (defined($next_hash_file = <$hash_files/*.pl>)) {
require $next_hash_file;
}

}
